In [6]:
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from datetime import datetime
from matplotlib.dates import date2num
import geopandas as gpd
from shapely.geometry import Point,Polygon
import plotly.graph_objects as go
import plotly.express as px
import plotly_express as px
import squarify
# PCA & MDS #
from sklearn import manifold
from sklearn.metrics import euclidean_distances
from sklearn.decomposition import PCA
from sklearn.manifold import MDS
from scipy.spatial.distance import pdist, cdist
In [40]:
df_gender=pd.read_csv("data on gender on england population.csv")
In [41]:
df_gender = df_gender.iloc[1: , :]
In [42]:
df_gender = df_gender.drop(['CDU_ID','GEO_TYP2'], axis=1)
In [222]:
df_gender
Out[222]:
In [520]:
df_gender['GEO_CODE']=='Shropshire'
Out[520]:
In [1089]:
Key = df_gender[df_gender['GEO_LABEL']=='Bedfordshire']
In [1095]:
Key = df_gender[df_gender['GEO_CODE']=='E06000055']
Berkshire = "West Berkshire" BRISTOL = "Bristol, City of" Cheshire = "Cheshire East" Cornwall = "Cornwall, Isles of Scilly" Derbyshire = "Derbyshire Dales" Devon = "Mid Devon" Durham = "County Durham" Herefordshire = "Herefordshire, County of" Kingston upon Hull = ""Kingston upon Hull, City of" Westminster = "City of London, Westminster" Stockton-on-Trent = "Stoke-on-Trent" Bedfordshire = "Central Bedfordshire" London = "City of London 001"
Bedfordshire = "Central Bedfordshire"
254,381 Central Bedfordshire E06000056¶
157,479 Bedford E06000055¶
In [1096]:
Key
Out[1096]:
In [614]:
["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
Out[614]:
In [ ]:
"E09000002",
Geocodes¶
Barking and Dagenham - E09000002
Bath and north east somerset - E06000022
Bedfordshire - E06000056
Berkshire - E06000037
Bexley - E09000004
Blackburn with Darwen - E06000008
Bournemouth - E06000028
Brent - E09000005
Brighton and Hove - E06000043
Bristol - E06000023
Bromley - E09000006
Buckinghamshire - E10000002
Cambridgeshire - E07000009
Camden - E09000007
Cheshire - E06000049
Cornwall - E06000052
Croydon - E09000008
Cumbria - E10000006
Darlington - E06000005
Derbyshire - E07000035
Derby - E06000015
Devon - E07000042
Dorset - E07000049
Durham - E06000047
Eailing- E09000009
East riding of Yorkshire - E06000011
East sussex - E07000228
Enfield - E09000010
Essex - E10000012
Gloucestershire - E06000025
Greenwich - E09000011
Hackney - E09000012
Halton - E06000006
Hammersmith and Fulham - E09000013
Hampshire - E07000085
Haringey - E09000014
Harrow - E09000015
Hartlepool - E06000001
Havering - E09000016
Herefordshire - E06000019
Hertfordshire - E07000099
Hillingdon - E09000017
Hounslow - E09000018
Isle of Wight - E06000046
Islington - E09000019
Kinsington and Chelsea - E09000020
Kent - E10000016
Kingston upon Hull - E06000010
Lambeth - E09000022
Lancashire - E07000127
Leicestershire - E07000134
Leicester - E06000016
Lewisham - E09000023
Lincolnshire - E06000013
City of London - E09000001
Luton - E06000032
Manchester - E08000003
Medway - E06000035
Merseyside - E11000002
Merton - E09000024
Middlesbrough - E06000002
Milton Keynes - E06000042
Newham - E09000025
Norfolk - E10000020
North East Lincolnshire - E06000012
North Somerset - E06000024
North Yorkshire - E10000023
Northamptonshire - E10000021
Northumberland - E06000048
Nottinghamshire - E10000024
Nottingham - E06000018
Oxfordshire - E10000025
Peterborough - E06000031
Plymouth - E06000026
Poole - E06000029
Portsmouth - E06000044
Redbridge - E09000026
Redcar and cleveland - E06000003
Richmond Upon Thames - E09000027
Rutland - E06000017
Shropshire - E06000051
Somerset - E10000027
South Gloucestershire - E06000025
South Yorkshire - E11000003
Southampton - E06000045
Southend-on-sea - E06000033
Southwark - E09000028
Staffordshire - E10000028
Stockton-on-tees - E06000004
Stoke-on-trent - E06000021
Suffolk - E10000029
Surrey - E10000030
Sutton - E09000029
Swindon - E06000030
Telford and wrekin - E06000020
Thurrock - E06000034
Torbay - E06000027
Tower Hamlets - E09000030
Tyne and Wear - E11000004
Waltham Forest - E09000031
Wandsworth - E09000032
Warrington - E06000007
Warwichshire - E10000031
West Midlands - E11000005
West Sussex - E10000032
West Yorkshire - E11000006
Westminster - E09000033
Wiltshire - E06000054
Worcesstershire - E10000034
York - E06000014
Kingston upon Thames - E09000021
Milton Keynes - E06000042
Colate all codes which corresponds to our shape files visualised on QGIS
E09000002,E06000022,E06000056,E06000037,E09000004,E06000008,E06000028,E09000005,E06000043,E06000023,E09000006,E10000002,E07000009,E09000007,E06000049,E06000052,E09000008,E10000006,E06000005,E07000035,E06000015,E07000042,E07000049,E06000047,E09000009,E06000011,E07000228,E09000010,E10000012,E06000025,E09000011,E09000012,E06000006,E09000013,E07000085,E09000014,E09000015,E06000001,E09000016,E06000019,E07000099,E09000017,E09000018,E06000046,E09000019,E09000020,E10000016,E06000010,E09000022,E07000127,E07000134,E06000016,E09000023,E06000013,E09000001,E06000032,E08000003,E06000035,E11000002,E09000024,E06000002,E06000042,E09000025,E10000020,E06000012,E06000024,E10000023,E10000021,E06000048,E10000024,E06000018,E10000025,E06000031,E06000026,E06000029,E06000044,E09000026,E06000003,E09000027,E06000017,E06000051,E10000027,E06000025,E11000003,E06000045,E06000033,E09000028,E10000028,E06000004,E06000021,E10000029,E10000030,E09000029,E06000030,E06000020,E06000034,E06000027,E09000030,E11000004,E09000031,E09000032,E06000007,E10000031,E11000005,E10000032,E11000006,E09000033,E06000054,E06000014,E09000021,E10000034
Filter Dataframe¶
In [615]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [616]:
filtered_Gender = df_gender[df_gender['GEO_CODE'].isin(GeoCodes)]
print(filtered_Gender )
In [618]:
filtered_Gender.head()
Out[618]:
Change name of countes to match shape files on QGIS¶
- Berkshire = "West Berkshire"
- BRISTOL = "Bristol, City of"
- Cheshire = "Cheshire East"
- Cornwall = "Cornwall, Isles of Scilly"
- Derbyshire = "Derbyshire Dales"
- Devon = "Mid Devon"
- Durham = "County Durham"
- Herefordshire = "Herefordshire, County of"
- Kingston upon Hull = ""Kingston upon Hull, City of"
- City of London = "City of London 001"
- Westminster = "City of London, Westminster"
- Stockton-on-Trent = "Stoke-on-Trent"
In [644]:
#change one name using key slice
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
In [1126]:
#Check if change of name works
Key = filtered_Gender[filtered_Gender['GEO_CODE']=='E06000021']
In [ ]:
In [1127]:
Key
Out[1127]:
In [ ]:
In [1113]:
#Change name of countes to match shape files on QGIS
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
filtered_Gender.loc[filtered_Gender.GEO_CODE == 'E06000056', 'GEO_LABEL'] = "Bedfordshire"
In [1003]:
filtered_Gender = filtered_Gender.drop(['GEO_TYPE'], axis=1)
In [1004]:
filtered_Gender.head()
Out[1004]:
In [960]:
# Transform the data into a numpy ndarray
np_array_gender = filtered_Gender.values
# Removing the first four columns
np_array_gender = np_array_gender[:, 3:]
# Building the PCA model
pca_gender = PCA(n_components=2).fit(np_array_gender) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_g = pca_gender.transform(np_array_gender) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_v variable has a shape of :", pca_x_g.shape)
print(pca_x_g)
In [961]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs 'Gender' ")
plt.scatter(pca_x_g[:,0], pca_x_g[:,1],cmap='viridis')
Out[961]:
In [ ]:
In [962]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_gender.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_Gender.columns[3+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_gender.components_[1])[np.argsort(np.abs(pca_gender.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_Gender.columns[3+np.argsort(np.abs(pca_gender.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(3):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(3):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [655]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Sex : Males - Unit : Persons', data=filtered_Gender)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [656]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Sex : Females - Unit : Persons', data=filtered_Gender)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [55]:
filtered_Gender = filtered_Gender.drop(['Sex : Total\ Sex - Unit : Persons'], axis=1)
In [660]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
gender_num = filtered_Gender.select_dtypes(include='number') # selects numeric columns only
gender_num.head()
Out[660]:
In [661]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat = cdist(gender_num, gender_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat.shape)
In [662]:
mds_gender = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_gender = mds_gender.fit_transform(dist_mat)
In [963]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_gender[:, 0], mds_x_gender[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_Gender['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_gender[index, 0], mds_x_gender[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
In [ ]:
Vehicle Accessibility per household in England Data¶
In [665]:
df_vehi=pd.read_csv("Vehicle accessibility per household in England Data.csv")
In [666]:
df_vehi = df_vehi.iloc[1: , :]
In [667]:
df_vehi
Out[667]:
In [668]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [669]:
filtered_vehi = df_vehi[df_vehi['GEO_CODE'].isin(GeoCodes)]
filtered_vehi.head()
Out[669]:
In [670]:
filtered_vehi = filtered_vehi.drop('CDU_ID', 1)
#filtered_vehi = filtered_vehi.drop('CDU_ID', 1)
In [671]:
filtered_vehi.corr()
Out[671]:
In [672]:
Car or van availability : Total\ Car or van availability - Unit : Households
Car or van availability : Total\ Car or van availability - Unit : Cars or vans
Car or van availability : No cars or vans in household - Unit : Households
Car or van availability : 1 car or van in household - Unit : Households
Car or van availability : 2 cars or vans in household - Unit : Households
Car or van availability : 3 cars or vans in household - Unit : Households
Car or van availability : 4 or more cars or vans in household - Unit : Households
Car or van availability : Sum of all cars or vans - Unit : Cars or vans
In [673]:
filtered_vehi.rename(columns = {'Car or van availability : Total\ Car or van availability - Unit : Households':'Total Number of households which have access to a Vehicle ',
'Car or van availability : Total\ Car or van availability - Unit : Cars or vans':'Total number of cars which are privately accessible to citizens(Owned/leased)',
'Car or van availability : No cars or vans in household - Unit : Households':'Total Number of households which do not have access to vehicles',
'Car or van availability : 1 car or van in household - Unit : Households':'Number of households which have access to 1 vehicle',
'Car or van availability : 2 cars or vans in household - Unit : Households':'Number of households which have access to 2 vehicles',
'Car or van availability : 3 cars or vans in household - Unit : Households':'Number of households which have accessibilty to 3 vehicles',
'Car or van availability : 4 or more cars or vans in household - Unit : Households':'Number of households which have access to 4 or more vehicles'}, inplace = True)
In [1005]:
filtered_vehi.head()
Out[1005]:
In [1006]:
filtered_vehi = filtered_vehi.drop(['GEO_TYPE'], axis=1)
In [1007]:
filtered_vehi = filtered_vehi.drop(['GEO_TYP2'], axis=1)
In [1008]:
filtered_vehi.head()
Out[1008]:
In [ ]:
In [675]:
filtered_vehi = filtered_vehi.drop('Car or van availability : Sum of all cars or vans - Unit : Cars or vans', 1)
In [676]:
filtered_vehi
Out[676]:
In [71]:
filtered_vehi.corr()
Out[71]:
In [964]:
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_vehi.loc[filtered_vehi.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [ ]:
In [677]:
# Transform the data into a numpy ndarray
np_array_vehi = filtered_vehi.values
In [678]:
# Removing the first four columns
np_array_vehi = np_array_vehi[:, 4:]
In [679]:
np_array_vehi
Out[679]:
In [680]:
# Building the PCA model
pca_vehi = PCA(n_components=2).fit(np_array_vehi) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_v = pca_vehi.transform(np_array_vehi) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_v variable has a shape of :", pca_x_v.shape)
print(pca_x_v)
In [681]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_v[:,0], pca_x_v[:,1],cmap='viridis')
Out[681]:
In [682]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_vehi.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_vehi.columns[4+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_vehi.components_[1])[np.argsort(np.abs(pca_vehi.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_vehi.columns[4+np.argsort(np.abs(pca_vehi.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(7):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(7):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [683]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Total Number of households which do not have access to vehicles', data=filtered_vehi)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [684]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Number of households which have access to 1 vehicle', data=filtered_vehi)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [685]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
vehi_num = filtered_vehi.select_dtypes(include='number') # selects numeric columns only
vehi_num.head()
Out[685]:
In [686]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat = cdist(vehi_num, vehi_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat.shape)
In [687]:
mds_vehi = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_vehi = mds_vehi.fit_transform(dist_mat)
In [965]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_vehi[:, 0], mds_x_vehi[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_vehi['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_vehi[index, 0], mds_x_vehi[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
In [ ]:
In [ ]:
In [ ]:
Fitting first to fourth order polynomials USE this when merging all PCA calculations¶
plt.figure(num=2, figsize = (16, 10)) plt.subplot(2,2,1) ax1 = sns.regplot(filtered_vehi['White %'], filtered_vehi['inEmployment'], order=1, scatter_kws={'alpha':0.3}, line_kws={'color':'tomato'}) ax1.set(title = 'Regression : Order 1') plt.subplot(2,2,2) ax2 = sns.regplot(filtered_vehi['Black %'], filtered_vehi['inEmployment'], order=1, scatter_kws={'alpha':0.3}, line_kws={'color':'tomato'}) ax2.set(title = 'Regression : Order 2') plt.subplot(2,2,3) ax3 = sns.regplot(filtered_vehi['Asian %'], filtered_vehi['inEmployment'], order=1, scatter_kws={'alpha':0.3}, line_kws={'color':'tomato'}) ax3.set(title = 'Regression : Order 3') plt.subplot(2,2,4) ax4 = sns.regplot(filtered_vehi['Mixed %'], filtered_vehi['inEmployment'], order=1, scatter_kws={'alpha':0.3}, line_kws={'color':'tomato'}) ax4.set(title = 'Regression : Order 4') plt.tight_layout();
In [ ]:
In [ ]:
Data on religion in England¶
In [689]:
df_reli=pd.read_csv("data on religion of England population .csv")
In [690]:
df_reli = df_reli.iloc[1: , :]
In [691]:
df_reli
Out[691]:
In [692]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [693]:
filtered_reli = df_reli[df_reli['GEO_CODE'].isin(GeoCodes)]
filtered_reli.head()
Out[693]:
In [966]:
filtered_reli.loc[filtered_reli.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_reli.loc[filtered_reli.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [1009]:
filtered_reli = filtered_reli.drop(['GEO_TYP2'], axis=1)
In [1010]:
filtered_reli = filtered_reli.drop(['GEO_TYPE'], axis=1)
In [1011]:
filtered_reli.head()
Out[1011]:
In [ ]:
In [694]:
# Transform the data into a numpy ndarray
np_array_reli = filtered_reli.values
In [695]:
# Removing the first four columns
np_array_reli = np_array_reli[:, 5:]
In [696]:
np_array_reli
Out[696]:
In [697]:
# Building the PCA model
pca_reli = PCA(n_components=2).fit(np_array_reli) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_r = pca_reli.transform(np_array_reli) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_r variable has a shape of :", pca_x_r.shape)
print(pca_x_r)
In [698]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_r[:,0], pca_x_r[:,1],cmap='viridis')
Out[698]:
In [699]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_reli.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_reli.columns[5+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_reli.components_[1])[np.argsort(np.abs(pca_reli.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_reli.columns[5+np.argsort(np.abs(pca_reli.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(7):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(7):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [700]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Religion [E][S][W] : Christian - Unit : Persons', data=filtered_reli)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [701]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Religion [E][S][W] : Muslim - Unit : Persons', data=filtered_reli)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [702]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Religion [E][S][W] : Hindu - Unit : Persons', data=filtered_reli)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [703]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Religion [E][S][W] : Sikh - Unit : Persons', data=filtered_reli)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [704]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Religion [E][S][W] : Total\ Religion - Unit : Persons', data=filtered_reli)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [705]:
# Change to a bit better style and larger figure.
sns.set_style('whitegrid')
fig, ax = plt.subplots(figsize=(35, 6))
# Plot our violins.
sns.violinplot(x='GEO_LABEL', y= 'Religion [E][S][W] : No religion - Unit : Persons', data=filtered_reli)
# Rotate the x-axis labels and remove the plot border on the left.
_ = plt.xticks(rotation=45, ha='right')
sns.despine(left=True)
In [ ]:
In [706]:
filtered_reli = filtered_reli.drop('CDU_ID', 1)
#filtered_vehi = filtered_vehi.drop('CDU_ID', 1)
In [707]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
reli_num = filtered_reli.select_dtypes(include='number') # selects numeric columns only
reli_num.head()
Out[707]:
In [708]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat_reli = cdist(reli_num, reli_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat_reli.shape)
In [709]:
mds_reli = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_reli = mds_reli.fit_transform(dist_mat_reli)
In [967]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_reli[:, 0], mds_x_reli[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_reli['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_reli[index, 0], mds_x_reli[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
In [ ]:
Data on population health England¶
In [711]:
df_health=pd.read_csv("Data on population health in England .csv")
df_health = df_health.iloc[1: , :]
In [712]:
df_health
Out[712]:
In [713]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [714]:
filtered_health = df_health[df_health['GEO_CODE'].isin(GeoCodes)]
filtered_health.head()
Out[714]:
In [715]:
filtered_health = filtered_health.drop(['General health : Total\ General health - Unit : Persons'], axis=1)
In [716]:
filtered_health
Out[716]:
In [968]:
filtered_health.loc[filtered_health.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_health.loc[filtered_health.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_health.loc[filtered_health.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_health.loc[filtered_health.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_health.loc[filtered_health.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_health.loc[filtered_health.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_health.loc[filtered_health.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_health.loc[filtered_health.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_health.loc[filtered_health.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_health.loc[filtered_health.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_health.loc[filtered_health.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_health.loc[filtered_health.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [1012]:
filtered_health = filtered_health.drop(['GEO_TYPE'], axis=1)
In [1013]:
filtered_health = filtered_health.drop(['GEO_TYP2'], axis=1)
In [1014]:
filtered_health.head()
Out[1014]:
In [ ]:
In [717]:
# Transform the data into a numpy ndarray
np_array_health = filtered_health.values
In [718]:
np_array_health
Out[718]:
In [719]:
# Removing the first four columns
np_array_health = np_array_health[:, 5:]
In [720]:
# Building the PCA model
pca_health = PCA(n_components=2).fit(np_array_health) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_h = pca_health.transform(np_array_health) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_h variable has a shape of :", pca_x_h.shape)
print(pca_x_h)
In [721]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_h[:,0], pca_x_h[:,1],cmap='viridis')
Out[721]:
In [ ]:
In [722]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_health.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_health.columns[5+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_health.components_[1])[np.argsort(np.abs(pca_health.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_health.columns[5+np.argsort(np.abs(pca_health.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(5):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(5):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [723]:
filtered_health = filtered_health.drop(['CDU_ID'], axis=1)
In [724]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
health_num = filtered_health.select_dtypes(include='number') # selects numeric columns only
health_num.head()
Out[724]:
In [725]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat_health = cdist(health_num, health_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat_health.shape)
In [726]:
mds_health = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_health = mds_health.fit_transform(dist_mat_health)
In [727]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_health[:, 0], mds_x_health[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_health['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_health[index, 0], mds_x_health[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
Data on Deprivation on UK counties & Boroughs¶
The dimensions of deprivation used to classify households are indicators based on four selected household characteristics. A household is deprived in a dimension if they meet one or more of the following conditions:
-
Employment: Where any member of a household, who is not a full-time student, is either unemployed or long-term sick.
Education: No person in the household has at least Level 2 education (see highest level of qualification), and no person aged 16 to 18 is a full-time student.
Health and disability: Any person in the household has general health that is 'bad' or 'very bad' or has a long-term health problem.
Housing: The household's accommodation is either overcrowded, with an occupancy rating -1 or less, or is in a shared dwelling, or has no central heating.
</ul>
A household is classified as being deprived in none, or one to four of these dimensions in any combination.
In [728]:
df_deprivation=pd.read_csv("Deprevation in England Population.csv")
df_deprivation = df_deprivation.iloc[1: , :]
In [729]:
df_deprivation.head()
Out[729]:
In [731]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [732]:
filtered_dep = df_deprivation[df_deprivation['GEO_CODE'].isin(GeoCodes)]
filtered_dep.head()
Out[732]:
In [ ]:
In [733]:
Deprivation; classification of household [E][S][W] : Total\ Classification of household deprivation - Unit : Households
Deprivation; classification of household [E][S][W] : Household is not deprived in any dimension - Unit : Households
Deprivation; classification of household [E][S][W] : Household is deprived in 1 dimension - Unit : Households
Deprivation; classification of household [E][S][W] : Household is deprived in 2 dimensions - Unit : Households
Deprivation; classification of household [E][S][W] : Household is deprived in 3 dimensions - Unit : Households
Deprivation; classification of household [E][S][W] : Household is deprived in 4 dimensions - Unit : Households
In [735]:
filtered_dep.rename(columns = {'Deprivation; classification of household [E][S][W] : Total\ Classification of household deprivation - Unit : Households':'Total households examined using deprivation indexes',
'Deprivation; classification of household [E][S][W] : Household is not deprived in any dimension - Unit : Households':'Number of households which are not deprived in any of the scrutinised dimensions',
'Deprivation; classification of household [E][S][W] : Household is deprived in 1 dimension - Unit : Households':'Number of households which are deprived by 1 of the scrutinised dimensions',
'Deprivation; classification of household [E][S][W] : Household is deprived in 2 dimensions - Unit : Households':'Number of households which are deprived by 2 of the scrutinised dimensions',
'Deprivation; classification of household [E][S][W] : Household is deprived in 3 dimensions - Unit : Households':'Number of households which are deprived by 3 of the scrutinised dimensions',
'Deprivation; classification of household [E][S][W] : Household is deprived in 4 dimensions - Unit : Households':'Number of households which are deprived by 4 of the scrutinised dimensions'}, inplace = True)
In [736]:
filtered_dep
Out[736]:
In [969]:
filtered_dep.loc[filtered_dep.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_dep.loc[filtered_dep.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [1015]:
filtered_dep
Out[1015]:
In [1016]:
filtered_dep = filtered_dep.drop(['GEO_TYPE'], axis=1)
In [1017]:
filtered_dep = filtered_dep.drop(['GEO_TYP2'], axis=1)
In [737]:
filtered_dep = filtered_dep.drop(['CDU_ID'], axis=1)
In [1019]:
filtered_dep.head()
Out[1019]:
In [ ]:
In [ ]:
In [738]:
# Transform the data into a numpy ndarray
np_array_dep = filtered_dep.values
In [739]:
np_array_dep
Out[739]:
In [740]:
# Removing the first four columns
np_array_dep = np_array_dep[:, 4:]
In [741]:
np_array_dep
Out[741]:
In [742]:
# Building the PCA model
pca_dep = PCA(n_components=2).fit(np_array_dep) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_d = pca_dep.transform(np_array_dep) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_h variable has a shape of :", pca_x_d.shape)
print(pca_x_d)
In [743]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_d[:,0], pca_x_d[:,1],cmap='viridis')
Out[743]:
In [744]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_dep.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_dep.columns[4+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_dep.components_[1])[np.argsort(np.abs(pca_dep.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_dep.columns[4+np.argsort(np.abs(pca_dep.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(6):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(6):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [745]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
dep_num = filtered_dep.select_dtypes(include='number') # selects numeric columns only
dep_num.head()
Out[745]:
In [746]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat_dep = cdist(dep_num, dep_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat_dep.shape)
In [747]:
mds_dep = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_dep = mds_dep.fit_transform(dist_mat_dep)
In [970]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_dep[:, 0], mds_x_dep[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_dep['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_dep[index, 0], mds_x_dep[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
In [ ]:
Unpaid Carers¶
Description: A person is a provider of unpaid care if they look after or give help or support to family members, friends, neighbours or others because of long-term physical or mental ill-health or disability, or problems related to old age. This does not include any activities as part of paid employment. No distinction is made about whether any care that a person provides is within their own household or outside the household, so no explicit link can be made about whether the care provided is for a person within the household who has poor general health or a long-term health problem or disability.
In [749]:
df_uncar=pd.read_csv("Data on Unpaid Carers.csv")
df_uncar = df_uncar.iloc[1: , :]
In [750]:
df_uncar.head()
Out[750]:
In [751]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [752]:
filtered_uncar = df_uncar[df_uncar['GEO_CODE'].isin(GeoCodes)]
filtered_uncar.head()
Out[752]:
In [753]:
filtered_uncar = filtered_uncar.drop(['CDU_ID'], axis=1)
In [754]:
# Transform the data into a numpy ndarray
np_array_uncar = filtered_uncar.values
In [755]:
np_array_uncar
Out[755]:
In [971]:
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_uncar.loc[filtered_uncar.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [1020]:
filtered_uncar = filtered_uncar.drop(['GEO_TYP2'], axis=1)
In [1021]:
filtered_uncar = filtered_uncar.drop(['GEO_TYPE'], axis=1)
In [1022]:
filtered_uncar.head()
Out[1022]:
In [ ]:
In [756]:
# Removing the first four columns
np_array_uncar = np_array_uncar[:, 4:]
In [757]:
# Building the PCA model
pca_uncar = PCA(n_components=2).fit(np_array_uncar) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_uncar = pca_uncar.transform(np_array_uncar) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_h variable has a shape of :", pca_x_uncar.shape)
print(pca_x_uncar)
In [758]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_uncar[:,0], pca_x_uncar[:,1],cmap='viridis')
Out[758]:
In [759]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_uncar.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_uncar.columns[4+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_uncar.components_[1])[np.argsort(np.abs(pca_uncar.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_uncar.columns[4+np.argsort(np.abs(pca_uncar.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(5):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(5):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [760]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
uncar_num = filtered_uncar.select_dtypes(include='number') # selects numeric columns only
uncar_num.head()
Out[760]:
In [761]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat_uncar = cdist(uncar_num, uncar_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat_uncar.shape)
In [762]:
mds_uncar = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_uncar = mds_uncar.fit_transform(dist_mat_uncar)
In [972]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_uncar[:, 0], mds_x_uncar[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_uncar['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_uncar[index, 0], mds_x_uncar[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
In [ ]:
Economic Activity¶
Economic activity relates to whether or not a person who was aged 16 and over was working or looking for work in the week before census. Rather than a simple indicator of whether or not someone was currently in employment, it provides a measure of whether or not a person was an active participant in the labour market.
A person's economic activity is derived from their 'activity last week'. This is an indicator of their status or availability for employment - whether employed, unemployed, or their status if not employed and not seeking employment. Additional information included in the economic activity classification is also derived from information about the number of hours a person works and their type of employment - whether employed or self-employed.
The census concept of economic activity is compatible with the standard for economic status defined by the <a href=http://www.ilo.org/global/statistics-and-databases/classifications/lang--en/index.htm" target="_blank">International Labour Organisation (ILO)</a>. It is one of a number of definitions used internationally to produce accurate and comparable statistics on employment
In [764]:
df_ea=pd.read_csv("Economic activity dataset in England .csv")
df_ea = df_ea.iloc[1: , :]
In [765]:
df_ea.head()
Out[765]:
In [766]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [767]:
filtered_ea = df_ea[df_ea['GEO_CODE'].isin(GeoCodes)]
filtered_ea.head()
Out[767]:
In [973]:
filtered_ea.loc[filtered_ea.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_ea.loc[filtered_ea.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [1023]:
filtered_ea = filtered_ea.drop(['GEO_TYP2'], axis=1)
In [1024]:
filtered_ea = filtered_ea.drop(['GEO_TYPE'], axis=1)
In [ ]:
In [768]:
filtered_ea = filtered_ea.drop(['CDU_ID'], axis=1)
In [769]:
# Transform the data into a numpy ndarray
np_array_ea = filtered_ea.values
In [770]:
np_array_ea
Out[770]:
In [771]:
# Removing the first four columns
np_array_ea = np_array_ea[:, 4:]
In [772]:
# Building the PCA model
pca_ea = PCA(n_components=2).fit(np_array_ea) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_ea = pca_ea.transform(np_array_ea) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_ea variable has a shape of :", pca_x_ea.shape)
print(pca_x_ea)
In [773]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_ea[:,0], pca_x_ea[:,1],cmap='viridis')
Out[773]:
In [775]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_ea.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_ea.columns[4+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_ea.components_[1])[np.argsort(np.abs(pca_ea.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_ea.columns[4+np.argsort(np.abs(pca_ea.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(5):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(5):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [776]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
ea_num = filtered_ea.select_dtypes(include='number') # selects numeric columns only
ea_num.head()
Out[776]:
In [777]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat_ea = cdist(ea_num, ea_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat_ea.shape)
In [778]:
mds_ea = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_ea = mds_ea.fit_transform(dist_mat_ea)
In [974]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_ea[:, 0], mds_x_ea[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_ea['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_ea[index, 0], mds_x_ea[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
Data info on England population¶
The main population base for statistics from the 2011 Census is the usual resident population as at census day, 27 March 2011. Although the population base for enumeration included non-UK born short-term residents, this population is analysed separately and is not included in the main outputs from the 2011 Census.
All statistics, unless specified, are produced using only usual residents of the UK.
For 2011 Census purposes, a usual resident of the UK is anyone who, on census day, was in the UK and had stayed or intended to stay in the UK for a period of 12 months or more, or had a permanent UK address and was outside the UK and intended to be outside the UK for less than 12 months.
For information about the main population base for statistics, how other population sub-groups are counted, and all variable definitions, see information about <a href=http://web.ons.gov.uk/ons/guide-method/census/2011/census-data/2011-census-user-guide/information-by-variable/index.html" target="_blank">variables and classifications</a>."
In [851]:
df_pop = pd.read_csv("Data on population in England.csv")
df_pop = df_pop.iloc[1: , :]
In [852]:
df_pop
Out[852]:
In [853]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [854]:
filtered_pop = df_pop[df_pop['GEO_CODE'].isin(GeoCodes)]
filtered_pop.head()
Out[854]:
Check for NaN values which is causing an error for our PCA calculation
In [855]:
filtered_pop.isnull().values.any()
Out[855]:
In [856]:
filtered_pop.isnull().sum().sum()
Out[856]:
In [857]:
count_nan_in_df = filtered_pop.isnull().sum()
print (count_nan_in_df)
In [858]:
df133 = filtered_pop[filtered_pop.isna().any(axis=1)]
In [859]:
#The two rows in the dataframes which have NaN values
df133
Out[859]:
We can fix this by calculating density (Population/Area)
In [860]:
Population_Density_Persons_Cornwall = (534476.0/356256.0)
Population_Density_Persons_Westminster = (226771.0/2438.0)
print(Population_Density_Persons_Cornwall)
print(Population_Density_Persons_Westminster)
We know that:
City of London, Westminster index = 110
&
Cornwall, Isles of Scilly = 109Therefore we use the iloc function to add:
- Population_Density_Persons_Cornwall
- Population_Density_Persons_Westminster
In [862]:
filtered_pop.update(filtered_pop.iloc[[109]].fillna('1.5002582412647085'))
filtered_pop.update(filtered_pop.iloc[[110]].fillna('93.01517637407711'))
In [863]:
filtered_pop.iloc[109]
Out[863]:
Done! Now lets continue
In [864]:
filtered_pop
Out[864]:
In [975]:
filtered_pop.loc[filtered_pop.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_pop.loc[filtered_pop.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [1025]:
filtered_pop = filtered_pop.drop(['GEO_TYP2'], axis=1)
In [1026]:
filtered_pop = filtered_pop.drop(['GEO_TYPE'], axis=1)
In [1027]:
filtered_pop.head()
Out[1027]:
In [865]:
filtered_pop = filtered_pop.drop(['CDU_ID'], axis=1)
In [866]:
# Transform the data into a numpy ndarray
np_array_pop = filtered_pop.values
In [867]:
np_array_pop
Out[867]:
In [868]:
# Removing the first four columns
np_array_pop = np_array_pop[:, 4:]
In [869]:
# Building the PCA model
pca_pop = PCA(n_components=2).fit(np_array_pop) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_pop = pca_pop.transform(np_array_pop) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_pop variable has a shape of :", pca_x_pop.shape)
print(pca_x_pop)
In [184]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_pop[:,0], pca_x_pop[:,1],cmap='viridis')
Out[184]:
In [185]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_pop.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_pop.columns[4+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_pop.components_[1])[np.argsort(np.abs(pca_pop.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_pop.columns[4+np.argsort(np.abs(pca_pop.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(7):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(7):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [870]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
pop_num = filtered_pop.select_dtypes(include='number') # selects numeric columns only
pop_num.head()
Out[870]:
In [871]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat_pop = cdist(pop_num, pop_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat_pop.shape)
In [872]:
mds_pop = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_pop = mds_pop.fit_transform(dist_mat_pop)
In [976]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_pop[:, 0], mds_x_pop[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_pop['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_pop[index, 0], mds_x_pop[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
Ethnicity Data on United Kingdom¶
In [874]:
df_ethnic=pd.read_csv("Data on ethnicity in England Geo.csv")
df_ethnic = df_ethnic.iloc[1: , :]
In [875]:
df_ethnic
Out[875]:
In [876]:
GeoCodes=["E09000002","E06000022","E06000055","E06000037","E09000004","E06000008","E06000028","E09000005","E06000043","E06000023","E09000006","E10000002","E10000003","E09000007","E06000049","E41000052","E09000008","E10000006","E06000005","E07000035","E06000015","E07000042","E10000009","E06000047","E09000009","E06000011","E10000011","E09000010","E10000012","E10000013","E09000011","E09000012","E06000006","E09000013","E10000014","E09000014","E09000015","E06000001","E09000016","E06000019","E10000015","E09000017","E09000018","E06000046","E09000019","E09000020","E10000016","E06000010","E09000021","E09000022","E10000017","E06000016","E10000018","E09000023","E10000019","E02000001","E06000032","E08000003","E06000035","E11000002","E09000024","E06000002","E06000042","E09000025","E10000020","E06000012","E06000013","E06000024","E10000023","E10000021","E06000048","E06000018","E10000024","E10000025","E06000031","E06000026","E06000029","E06000044","E09000026","E06000003","E09000027","E06000017","E06000051","E10000027","E06000025","E11000003","E06000045","E06000033","E09000028","E10000028","E06000021","E06000004","E10000029","E10000030","E09000029","E06000030","E06000020","E06000034","E06000027","E09000030","E11000004","E09000031","E09000032","E06000007","E10000031","E11000005","E10000032","E11000006","E41000324","E06000054","E10000034","E06000014"]
In [877]:
filtered_ethnic = df_ethnic[df_ethnic['GEO_CODE'].isin(GeoCodes)]
print(filtered_ethnic.head())
In [878]:
filtered_ethnic.columns
Out[878]:
In [ ]:
In [880]:
filtered_ethnic.rename(columns = {'Ethnic group [E][S][W] : Total\ Ethnic group - Unit : Persons':'Total\ Ethnic group - Unit : Persons',
'Ethnic group [E][S][W] : White\ English/Welsh/Scottish/Northern Irish/British - Unit : Persons':'White\ English/Welsh/Scottish/Northern Irish/British - Unit : Persons',
'Ethnic group [E][S][W] : White\ Irish - Unit : Persons':'White\ Irish - Unit : Persons',
'Ethnic group [E][S][W] : White\ Gypsy or Irish Traveller - Unit : Persons':'White\ Gypsy or Irish Traveller - Unit : Persons',
'Ethnic group [E][S][W] : White\ Other White - Unit : Persons':'White\ Other White - Unit : Persons',
'Ethnic group [E][S][W] : Mixed/multiple ethnic group\ White and Black Caribbean - Unit : Persons':'Mixed/multiple ethnic group\ White and Black Caribbean - Unit : Persons',
'Ethnic group [E][S][W] : Mixed/multiple ethnic group\ White and Black African - Unit : Persons':'Mixed/multiple ethnic group\ White and Black African - Unit : Persons',
'Ethnic group [E][S][W] : Mixed/multiple ethnic group\ White and Asian - Unit : Persons':'Mixed/multiple ethnic group\ White and Asian - Unit : Persons',
'Ethnic group [E][S][W] : Mixed/multiple ethnic group\ Other Mixed - Unit : Persons':'Mixed/multiple ethnic group\ Other Mixed - Unit : Persons',
'Ethnic group [E][S][W] : Asian/Asian British\ Indian - Unit : Persons':'Asian/Asian British\ Indian - Unit : Persons',
'Ethnic group [E][S][W] : Asian/Asian British\ Pakistani - Unit : Persons':'Asian/Asian British\ Pakistani - Unit : Persons',
'Ethnic group [E][S][W] : Asian/Asian British\ Bangladeshi - Unit : Persons':'Asian/Asian British\ Bangladeshi - Unit : Persons',
'Ethnic group [E][S][W] : Asian/Asian British\ Chinese - Unit : Persons':'Asian/Asian British\ Chinese - Unit : Persons',
'Ethnic group [E][S][W] : Asian/Asian British\ Other Asian - Unit : Persons':'Asian/Asian British\ Other Asian - Unit : Persons',
'Ethnic group [E][S][W] : Black/African/Caribbean/Black British\ African - Unit : Persons':'Black/African/Caribbean/Black British\ African - Unit : Persons',
'Ethnic group [E][S][W] : Black/African/Caribbean/Black British\ Caribbean - Unit : Persons':'Black/African/Caribbean/Black British\ Caribbean - Unit : Persons',
'Ethnic group [E][S][W] : Black/African/Caribbean/Black British\ Other Black - Unit : Persons':'Black/African/Caribbean/Black British\ Other Black - Unit : Persons',
'Ethnic group [E][S][W] : Other ethnic group\ Arab - Unit : Persons':'Number of people classified as Arab ethnic group',
'Ethnic group [E][S][W] : Other ethnic group\ Any other ethnic group - Unit : Persons':'Other ethnic group\ Any other ethnic group - Unit : Persons'}, inplace = True)
In [881]:
filtered_ethnic
Out[881]:
In [977]:
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E06000037', 'GEO_LABEL'] = "Berkshire"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E06000023', 'GEO_LABEL'] = "Bristol"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E06000049', 'GEO_LABEL'] = "Cheshire"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E41000052', 'GEO_LABEL'] = "Cornwall"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E07000035', 'GEO_LABEL'] = "Derbyshire"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E07000042', 'GEO_LABEL'] = "Devon"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E06000047', 'GEO_LABEL'] = "Durham"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E06000019', 'GEO_LABEL'] = "Herefordshire"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E06000010', 'GEO_LABEL'] = "Kingston upon Hull"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "City of London"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E41000324', 'GEO_LABEL'] = "Westminster"
filtered_ethnic.loc[filtered_ethnic.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stockton-on-Trent"
In [1028]:
filtered_ethnic = filtered_ethnic.drop(['GEO_TYPE'], axis=1)
In [1029]:
filtered_ethnic = filtered_ethnic.drop(['GEO_TYP2'], axis=1)
In [ ]:
In [882]:
filtered_ethnic = filtered_ethnic.drop(['CDU_ID'], axis=1)
In [883]:
# Transform the data into a numpy ndarray
np_array_ethnic = filtered_ethnic.values
In [884]:
np_array_ethnic
Out[884]:
In [ ]:
In [885]:
# Removing the first four columns
np_array_ethnic = np_array_ethnic[:, 4:]
In [886]:
# Building the PCA model
pca_ethnic = PCA(n_components=2).fit(np_array_ethnic) # We're telling PCA here to keep only the 2 first principal components when fitting the data
pca_x_ethnic = pca_ethnic.transform(np_array_ethnic) # This variable will contain the coordinates of each row in the new system of coordinates found by PCA (restricted to the first two directions or principal components)
# Those 2 Principal Components are going to be used to project the data onto a 2D space in order to be able
# to visualize a "compressed" version (in terms of variance) of our initial multivariate DataFrame
# Let's check out the result of our PCA applied to the dataFrame df
print("The pca_x_ethnic variable has a shape of :", pca_x_ethnic.shape)
print(pca_x_ethnic)
In [887]:
plt.subplot(1,2,1)
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("Projection of the original data onto the first 2 PCs")
plt.scatter(pca_x_ethnic[:,0], pca_x_ethnic[:,1],cmap='viridis')
Out[887]:
In [888]:
# Ordering the features by their coefficients in the equation of the first Principal Component
# PC1 = a*column1 + b*column2 + ... + zzz*column101. The coefficients of PC1 a, b, c, ..zzz also called loadings, are stored in pca_clean.components_[0]
pca_x_1 = pca_ethnic.components_[0] # First principal component
sort_array_1 = np.argsort(np.abs(pca_x_1)) # Sorting the 'Indexes' of the pca_clean_1 array from the lowest value to the highest (in magnitude, ignoring the sign)
pca_x_1_highest_values = pca_x_1[sort_array_1][::-1][:10] # We sorted from low to high, [::-1] retrieves all values backward (to inverse the sort), [:10] get the first
pca_x_1_cols_highest_values = filtered_ethnic.columns[4+sort_array_1][::-1][:10] # retrieving the names of the columns corresponding to the highest values
# The '1+sort_array' accounts for the fact that df has the 'communityname' column and sort_array sorted indices in an array that didn't have the column (we dropped it before doing PCA)
# Same thing but ... a little bit more compact
pca_x_2_highest_values = np.asarray(pca_ethnic.components_[1])[np.argsort(np.abs(pca_ethnic.components_[1]))][::-1][:10]
pca_x_2_cols_highest_values = filtered_ethnic.columns[4+np.argsort(np.abs(pca_ethnic.components_[1]))][::-1][:10]
# Printing the result
print('\nFirst Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(10):
print('The Column "{}" has a loading of : {}'.format(pca_x_1_cols_highest_values[i], pca_x_1_highest_values[i]))
print('\nSecond Principal Component : 10 Highest Loadings (in magnitude)\n')
for i in range(10):
print('The Column "{}" has a loading of : {}'.format(pca_x_2_cols_highest_values[i], pca_x_2_highest_values[i]))
In [889]:
# Let's extract the numeric columns and turn them into a numpy 2d array (matrix)
ethnic_num = filtered_ethnic.select_dtypes(include='number') # selects numeric columns only
ethnic_num.head()
Out[889]:
In [890]:
# Let's compute the distance matrix of pairwise euclidean distances
dist_mat_ethnic = cdist(ethnic_num, ethnic_num) # computes the distances between rows of first dataframe and rows of second dataframe. Here I gave the same dataFrame twice, to get a pairwise distance matrix
print("The shape of this matrix is", dist_mat_ethnic.shape)
In [891]:
mds_ethnic = MDS(n_components=2, random_state=123, dissimilarity='precomputed') # dissimilarity precomputed means that we provide the distance matrix, otherwise MDS() can compute it itself
mds_x_ethnic = mds_ethnic.fit_transform(dist_mat_ethnic)
In [978]:
# Let's visualize the results
plt.figure(num=5, figsize=(25, 12))
plt.scatter(mds_x_ethnic[:, 0], mds_x_ethnic[:, 1], alpha = .3, s=100)
# We need to loop over every borough, get its name from the borough dataframe and its location from mds_x and add the text to our scatterplot
for index,name in enumerate(filtered_ethnic['GEO_LABEL']):
plt.annotate(s=name, xy=(mds_x_ethnic[index, 0], mds_x_ethnic[index, 1]))
plt.xlabel('Axis 1')
plt.ylabel('Axis 2')
plt.title('Multidimensional Scaling on Districts and subdivisions (All rows)');
In [ ]:
Merging Datasets & Save datasets¶
Review:
Gender = filtered_Gender
Vehicle accessibility = filtered_vehi
Religion = filtered_reli
health = filtered_health
deprivation = filtered_dep
unpaid carers = filtered_uncar
economic activity = filtered_ea
population = filtered_pop
ethnicity = filtered_ethnic
- Gender - (filtered_Gender)
- Vehicle Accessiblity - (filtered_vehi)
- religion - (filtered_reli)
- health - (filtered_health)
- deprivation - (filtered_dep)
- unpaid carers - (filtered_uncar)
- economic activity - (filtered_ea)
- Population - (filtered_pop)
- ethnic - (filtered_ethnic)
Lets check all shapes of dataframe
In [1080]:
# saving the filtered_gender
filtered_Gender.to_csv('filtered_gender_england_dataset.csv', header=True, index=False)
filtered_vehi.to_csv('filtered_vehicle_access_england_dataset.csv', header=True, index=False)
filtered_reli.to_csv('filtered_religion_england_dataset.csv', header=True, index=False)
filtered_health.to_csv('filtered_health_england_dataset.csv', header=True, index=False)
filtered_dep.to_csv('filtered_deprivation_england_dataset.csv', header=True, index=False)
filtered_uncar.to_csv('filtered_unpaid_carers_england_dataset.csv', header=True, index=False)
filtered_ea.to_csv('filtered_econ_activity_england_dataset.csv', header=True, index=False)
filtered_pop.to_csv('filtered_population_england_dataset.csv', header=True, index=False)
filtered_ethnic.to_csv('filtered_ethnic_england_dataset.csv', header=True, index=False)
In [908]:
filtered_Gender.shape
Out[908]:
In [909]:
filtered_vehi.shape
Out[909]:
In [910]:
filtered_reli.shape
Out[910]:
In [911]:
filtered_health.shape
Out[911]:
In [912]:
filtered_dep.shape
Out[912]:
In [913]:
filtered_uncar.shape
Out[913]:
In [914]:
filtered_ea.shape
Out[914]:
In [916]:
filtered_pop.shape
Out[916]:
In [917]:
filtered_ethnic.shape
Out[917]:
In [ ]:
In [ ]:
#Data on crime commited in the city of London between 03-06(lockdown) merged
df_crime_all=[crime_oct2019,crime_Nov2019,crime_dec2019,crime_jan2020,crime_feb2020,crime_march2020,crime_april2020,crime_may2020,crime_june2020,crime_july2020,crime_august2020,crime_Sep2020,crime_oct2020];
df_crime= pd.concat(df_crime_all , axis=0, join='outer',ignore_index=True, keys=None,
levels=None, names=None, verify_integrity=False, copy=True)
In [ ]:
df_all123 = pd.concat(df_all,axis=0,join='outer',ignore_index=True,keys=None,levels=None,names=None,verify_integrity=True,copy=False)
In [1051]:
df_test=pd.concat(df_all,axis=1)
In [1057]:
df_genvehi=pd.merge(filtered_Gender,filtered_vehi)
#df_all=[filtered_Gender,filtered_vehi,filtered_reli,filtered_health,filtered_dep,filtered_uncar,filtered_ea,filtered_pop,filtered_ethnic];
In [1060]:
df_genvehi.head()
Out[1060]:
In [1058]:
df_3=pd.merge(df_genvehi,filtered_reli)
In [1059]:
df_3.head()
Out[1059]:
In [1062]:
df_4=pd.merge(df_3,filtered_health)
In [1063]:
df_4.head()
Out[1063]:
In [1064]:
df_5=pd.merge(df_4,filtered_dep)
In [1065]:
df_5.head()
Out[1065]:
In [1066]:
df_6=pd.merge(df_5,filtered_ea)
In [1067]:
df_6.head()
Out[1067]:
In [1068]:
df_7=pd.merge(df_6,filtered_pop)
In [1069]:
df_7.head()
Out[1069]:
In [1070]:
df_8=pd.merge(df_7,filtered_uncar)
In [1071]:
df_8.head()
Out[1071]:
In [1072]:
df_9=pd.merge(df_8,filtered_ethnic)
In [1073]:
df_9
Out[1073]:
In [1124]:
Key = df_9[df_9['GEO_CODE']=='E02000001']
In [1125]:
Key
Out[1125]:
In [ ]:
In [1130]:
df_9.isnull().values.any()
Out[1130]:
In [1131]:
#last two changes needed to be made
df_9.loc[df_9.GEO_CODE == 'E06000055', 'GEO_LABEL'] = "Bedfordshire"
df_9.loc[df_9.GEO_CODE == 'E02000001', 'GEO_LABEL'] = "London"
df_9.loc[df_9.GEO_CODE == 'E06000021', 'GEO_LABEL'] = "Stoke-on-Trent"
In [1132]:
df_9.to_csv('Combined_England_dataset_Engineered.csv', header=True, index=False)
In [7]:
Load=pd.read_csv('Combined_England_dataset_Engineered.csv')
In [8]:
Load
Out[8]:
In [ ]:
In [ ]:
In [ ]: